{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.special"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork:\n",
    "    # initialise the neural network\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate) -> None:\n",
    "        self.inodes = input_nodes\n",
    "        self.hnodes = hidden_nodes\n",
    "        self.onodes = output_nodes\n",
    "        self.lr = learning_rate\n",
    "\n",
    "        # set weight: input -> hidden, -0.5 ~ +0.5\n",
    "        # self.wih = np.random.rand(self.hnodes, self.inodes) - 0.5\n",
    "        self.wih = np.random.normal(0.0, pow(self.hnodes, -0.5), (self.hnodes, self.inodes))\n",
    "        # hidden -> output\n",
    "        # self.who = np.random.rand(self.onode, self.hnodes) - 0.5\n",
    "        self.who = np.random.normal(0.0, pow(self.onodes, -0.5), (self.onodes, self.hnodes))\n",
    "\n",
    "        # activation function: sigmoid = 1/(1+exp(-x))\n",
    "        self.activation_function = lambda x: scipy.special.expit(x)\n",
    "\n",
    "    # train the neural network\n",
    "    def train(self, input_list, target_list):\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        targets = np.array(target_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "\n",
    "        # calculate errors\n",
    "        output_errors = targets - final_outputs\n",
    "        hidden_errors = np.dot(self.who.T, output_errors)\n",
    "\n",
    "        # update weights\n",
    "        self.who += self.lr * np.dot(output_errors * final_outputs * (1.0 - final_outputs), np.transpose(hidden_outputs))\n",
    "        self.wih += self.lr * np.dot(hidden_errors * hidden_outputs * (1.0 - hidden_outputs), np.transpose(inputs))\n",
    "\n",
    "    # query the neural network\n",
    "    def query(self, input_list):\n",
    "        # convert input to 2d array\n",
    "        inputs = np.array(input_list, ndmin=2).T\n",
    "        hidden_inputs = np.dot(self.wih, inputs)\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        final_inputs = np.dot(self.who, hidden_outputs)\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "        return final_outputs\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.3\n",
    "input_nodes = 784 # image is 28 x 28\n",
    "hidden_nodes = 100\n",
    "output_nodes = 10 # 0 ~ 9 total 10 labels\n",
    "n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)\n",
    "# n.query([1.0, 0.5, -1.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_file = open('mnist_dataset/mnist_train_100.csv', 'r')\n",
    "training_data_list = training_data_file.readlines()\n",
    "training_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "for record in training_data_list:\n",
    "    all_values = record.split(',')\n",
    "    inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "    targets = np.zeros(output_nodes)\n",
    "    targets[int(all_values[0])] = 0.99\n",
    "    n.train(inputs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1.2266350125435805, 1.2073901897757846)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.who.min(), n.who.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.4939940343155577, 0.5303555124030355)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.wih.min(), n.wih.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test results\n",
    "test_data_file = open('mnist_dataset/mnist_test_10.csv', 'r')\n",
    "test_data_list = test_data_file.readlines()\n",
    "test_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_values = test_data_list[0].split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0464305 ],\n",
       "       [0.011314  ],\n",
       "       [0.00855409],\n",
       "       [0.04457144],\n",
       "       [0.06092862],\n",
       "       [0.02232578],\n",
       "       [0.00282243],\n",
       "       [0.84725175],\n",
       "       [0.01953903],\n",
       "       [0.00963694]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicts = n.query(np.asfarray(test_values[1:]) / 255.0 * 0.99 + 0.01)\n",
    "predicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'7'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "correct label: 7 predict label: 7\n",
      "correct label: 2 predict label: 0\n",
      "correct label: 1 predict label: 1\n",
      "correct label: 0 predict label: 0\n",
      "correct label: 4 predict label: 4\n",
      "correct label: 1 predict label: 1\n",
      "correct label: 4 predict label: 4\n",
      "correct label: 9 predict label: 3\n",
      "correct label: 5 predict label: 1\n",
      "correct label: 9 predict label: 7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1, 0, 1, 1, 1, 1, 1, 0, 0, 0]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# record scores\n",
    "score_card = []\n",
    "for record in test_data_list:\n",
    "    all_values = record.split(',')\n",
    "    correct_label = int(all_values[0])\n",
    "    print(f'correct label: {correct_label}', end=' ')\n",
    "    inputs = np.asfarray(all_values[1:]) / 255.0 * 0.99 + 0.01\n",
    "    outputs = n.query(inputs)\n",
    "    # get index of max value \n",
    "    label = np.argmax(outputs)\n",
    "    print(f'predict label: {label}')\n",
    "    if label == correct_label:\n",
    "        score_card.append(1)\n",
    "    else:\n",
    "        score_card.append(0)\n",
    "score_card"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_card_array = np.asarray(score_card)\n",
    "score_card_array.sum() / score_card_array.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('ai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0cb0046d74bfb5a9fee6ebb85b9850d762e40d6b321d5d9a0aa0313eefa3c5b4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
